package top.igotcha.hdfs;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.jupiter.api.Test;

import java.net.URI;

/**
 * @time 2021/1/10
 * @auth Gotcha
 * @describe
 */
public class HdfsMergeFile {
    //小文件合并
    @Test
    public void mergeFile() throws  Exception{
        //1.获取FileSystem文件系统
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.127.110:8020"), new Configuration(),"hadoop");
        //2.在HDFS获取要输出的文件流
        FSDataOutputStream outputStream = fileSystem.create(new Path("/bigfile.txt"));
        //3.获取LocalFileSystem本地文件系统
        LocalFileSystem local = FileSystem.getLocal(new Configuration());
        //4.获取通过本地文件集合
        FileStatus[] fileStatuses = local.listStatus(new Path("..\\hdfs_api\\temp"));
        //5.遍历集合进行合并后上传
        for (FileStatus fileStatus : fileStatuses) {
            FSDataInputStream inputStream = local.open(fileStatus.getPath());
            IOUtils.copy(inputStream,outputStream);
            IOUtils.closeQuietly(inputStream);
        }
        //6.关闭资源
        IOUtils.closeQuietly(outputStream);
        local.close();
        fileSystem.close();
    }
}
